package com.asiainfo.kuai;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import cn.edu.hfut.dmic.webcollector.util.ExceptionUtils;
import com.asiainfo.thepaper.ThepaperIndex;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;

/**
 * KuaiIndex
 * 快资讯财经首页：https://www.360kuai.com/pc/home/next?ch=y2%3Afinance&uid=0ab771384332edabdfd37eec452d3df8&sign=look
 *
 * @Author jhr
 * @Date 2022/4/12
 */

public class KuaiIndex extends BreadthCrawler {

    private static final Logger logger = LoggerFactory.getLogger(KuaiIndex.class);
    private static final String crawlPath = "/Users/jhr/data/db/KuaiIndex";
    //   要爬取的首页
    public static String seed = "";
    //    查找匹配的网站
    public static String RegularUrl = "";

    /**
     * 构造一个基于伯克利DB的爬虫
     * 伯克利DB文件夹为crawlPath，crawlPath中维护了历史URL等信息
     * 不同任务不要使用相同的crawlPath
     * 两个使用相同crawlPath的爬虫并行爬取会产生错误
     */
    public KuaiIndex(String Seed, String regularUrl) {
        super(crawlPath, false);
        seed = Seed;
        RegularUrl = regularUrl;
//        crawlPath = CrawlPath;
        CrawlDatum crawlDatum = new CrawlDatum(seed).meta("depth", "1");
        addSeed(crawlDatum);
        this.addRegex(RegularUrl);
        setThreads(1);
    }

    @Override
    public void visit(Page page, CrawlDatums next) {
        try {
            URL Url = new URL(page.url());
            HttpURLConnection conn = (HttpURLConnection) Url.openConnection();
            conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
        } catch (Exception e) {
            e.printStackTrace();
        }
        if (page.matchUrl(RegularUrl)) {
//            System.out.println(page.url());
            /*对重定向进行处理
             *不处理重定向请求,澎湃新闻中，会重定向至新华网：
             *例:打开此链接后 https://www.thepaper.cn/newsDetail_forward_17558091
             *会重定向至：https://h.xinhuaxmt.com/vh512/share/10715794
             */
            if (page.code() == 301 || page.code() == 302) {
                try {
                    // page.location() may be relative url path
                    // we have to construct an absolute url path
                    String redirectUrl = new URL(new URL(page.url()), page.doc().location()).toExternalForm();
                    next.addAndReturn(redirectUrl).meta(page.meta());
                } catch (MalformedURLException e) {
                    //the way to handle exceptions in WebCollector
                    ExceptionUtils.fail(e);
                }
                return ;
            }
            System.out.println("this page is not redirected: " + page.url());
            System.out.println("爬取网址：" + page.url());
            /*
            title:新闻标题
            content:新闻内容
            releaseTime:新闻发布时间
            author: 新闻作者
            source: 新闻来源
            */
            String title = "";
            String content = "";
            String releaseTime = "";
            String author = "";
            String source = "快资讯";
            //title 获取新闻标题
            Elements titleElements = page.select("h1.news_title");
            if (titleElements.size() > 0) {
                title = titleElements.get(0).text();
            } else {
                title = page.select("div.video_txt_t").text();
            }
            if (title.equals("")) {
                title = page.select("div.video_txt_t").text();
            }
            System.out.println("文章标题：" + title);

//            content 获取新闻内容
            Elements contentElements = page.select("div.news_txt");
            if (contentElements.size() > 0) {
                content = contentElements.text();
            } else {
                content = page.select("div.video_txt_l").text();
            }
            if (content.equals("")) {
                content = page.select("div.video_txt_l").text();
            }
            System.out.println("文章内容：" + content);

//            author 获取新闻作者
            Elements elements = page.select("div.news_about>p");
//            遇到视频类文章，分类处理
            if (elements.size() > 0) {
                author = elements.get(0).text();
            } else {
                //视频类型网站
                author = page.select("div.video_info_second>span").text();
            }
//        数据为空处理
            if (elements.text().equals("")) {
                author = page.select("div.name").text();
            }
            if (author.length() == 0) {
                author = page.select("div.video_info_second").text();
            }
            System.out.println("作者:" + author);

//            releaseTime 获取新闻发布时间
            if (elements.size() > 0) {
                releaseTime = elements.get(1).text();
            } else {
                //视频类型网站
                Elements timeEles = page.select("video_info_left");
                if (timeEles.size() > 0) {
                    releaseTime = timeEles.get(0).text();
                    if (timeEles.text().contains("来源：")) {
                        source = timeEles.get(0).text().split("来源：")[1];
                    }
                }
            }
            if (releaseTime.equals("")) {
                releaseTime = page.select("div.video_info_left>span").text();
            }
            if (releaseTime.contains("来源：")) {
//          source = releaseTime.substring(releaseTime.indexOf("来源") + 2);
                String[] strings = releaseTime.split("来源：");
                System.out.println("发布时间：" + strings[0]);
                if (strings[1].contains(" ")) {
                    source = strings[1].split(" ")[0];
                } else {
                    source = strings[1];
                }
            } else {
                System.out.println("发布时间：" + releaseTime);
            }
            System.out.println("来源：" + source);
//            无阅读量
        } else {
//            Elements elements = page.select("div.news_li>h2>a");
            Elements elements = page.select("a");
            System.out.println("获取到链接个数：" + elements.size());
            for (Element element : elements) {
                logger.debug("获取到匹配的url=" + element.attr("abs:href"));
                String regRx = RegularUrl;
//                匹配符合正则的网站
                if (element.attr("abs:href").matches(regRx)) {
                    CrawlDatum meta = new CrawlDatum(element.attr("abs:href")).meta("depth", "1").meta("refer", seed);
                    next.add(meta);
                } else {
                    System.out.println("正则URL不匹配！！！");
                }
            }
        }
        return ;
    }

    public static void main(String[] args) throws Exception {
//        String seed = "https://www.360kuai.com/pc/home/next?ch=y2%3Afinance&uid=0ab771384332edabdfd37eec452d3df8&sign=look";
//        财经>股票
        String seed = "https://www.360kuai.com/pc/home/next?ch=y2%3Afinance&uid=0ab771384332edabdfd37eec452d3df8&sign=look";
        String regex = "https://www.360kuai.com/9.*";
        KuaiIndex kuaiIndex = new KuaiIndex(seed, regex);
        kuaiIndex.start(2);
    }
}
